#!/usr/bin/env perl
use warnings;
use strict;
use FindBin qw/$Bin/;
use Net::FTP;
use Getopt::Long;

########################################################################
#v1.0 功能：下载cnv注释相关数据库；                      
#Date:2021-03-12
########################################################################
#Help:
my ($dgv,$decipher,$dbvar,$clingene,$all,$odir);
GetOptions(
                "help|?"     =>\&USAGE,
                "dgv:s"      =>\$dgv,
                "decipher:s" =>\$decipher,
                "dbvar:s"    =>\$dbvar,
                "clingene:s" =>\$clingene,
                "all:s"      =>\$all,
                "odir:s"      =>\$odir,
) or USAGE();

USAGE() unless (defined $dgv or defined $decipher or defined $dbvar or defined $clingene or defined $all);
#=======================================================================
#=======================================================================
my $TT0 = time();

#需要下载的文件列表和host/url
my ($cnv_filesHR,$hostsHR) = db_need_download();

#创建输出目录
my ($decipherDir,$dbvarDir,$dgvDir,$clingeneDir) = setOutDir();

#判断需要下载的数据库
if (defined $all){

    #下载dgv数据库
    download_dgv($hostsHR, $cnv_filesHR,$dgvDir);

    #下载解析decipher数据库
    system("/usr/bin/python3 $Bin/decipher_crawler_v2.py $decipherDir");

    #下载并解析dbvar数据库
    download_dbvar($hostsHR, $cnv_filesHR, $dbvarDir);

    #下载解析clingene数据库
    download_clingene($hostsHR, $cnv_filesHR, $clingeneDir);

}else{

    if (defined $dgv){

        #下载dgv数据库 文件200多M
        download_dgv($hostsHR, $cnv_filesHR,$dgvDir);
    }
    if (defined $decipher){

        #下载解析decipher数据库
        system("/usr/bin/python3 $Bin/decipher_crawler_v2.py $decipherDir");
    }
    if (defined $dbvar){

        #下载解析dbvar数据库
        download_dbvar($hostsHR, $cnv_filesHR, $dbvarDir);
    }
    if (defined $clingene){

        #下载解析clingene数据库
        download_clingene($hostsHR, $cnv_filesHR, $clingeneDir);
    }
}

#时间消耗
my $TT1=time();
my $mintime=sprintf("%.3f",($TT1-$TT0)/60);
print "$0-程序运行时间：$mintime分钟\n";
exit();
########################################################################
#定义sub                       
########################################################################
sub db_need_download
{
    my $cnv_filesHR ={
        'dgv'      => 'GRCh37_hg19_variants_2016-05-15.txt',
        'decipher' => 'Decipher_CNV.all.xls',
        'dbvarURL' => { 'nstd101' => 'https://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/Homo_sapiens/by_study/csv/all_variants_for_nstd101.csv.gz',
                        'nstd102' => 'https://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/Homo_sapiens/by_study/csv/all_variants_for_nstd102.csv.gz',
                        'nstd37'  => 'https://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/Homo_sapiens/by_study/csv/all_variants_for_nstd37.csv.gz',
                        'nstd45'  => 'https://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/Homo_sapiens/by_study/csv/all_variants_for_nstd45.csv.gz',
                        'nstd51'  => 'https://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/Homo_sapiens/by_study/csv/all_variants_for_nstd51.csv.gz',
                        'dbvar_mapping' => "https://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/release_notes/nstd102_accession_mapping.txt",
                      },
        'dbvar'    => { 'nstd101' => 'all_variants_for_nstd101.csv',
                        'nstd102' => 'all_variants_for_nstd102.csv',
                        'nstd37'  => 'all_variants_for_nstd37.csv',
                        'nstd45'  => 'all_variants_for_nstd45.csv',
                        'nstd51'  => 'all_variants_for_nstd51.csv',
                        'dbvar_mapping' => "nstd102_accession_mapping.txt",
                      },
        'syndrome' => 'Decipher_syndromes.all.xls',
        'isca'     => 'https://ftp.clinicalgenome.org/ClinGen_region_curation_list_GRCh37.tsv',
        'cyto'     => 'cytoBand.txt',
        'refGenePosi'   => 'refGene.txt',
        'geneInfo'      => 'gene_info_list.txt',
        'geneHTScore'   => 'https://ftp.clinicalgenome.org/ClinGen_gene_curation_list_GRCh37.tsv',
        'clinicalMatch' => 'Clinical_Match_finally.txt',
        'dbvar_mapping' => 'nstd102_accession_mapping.txt',
    };

    my $hostsHR = {
        'clingene' => 'ftp.clinicalgenome.org',
        'dbvar' => 'ftp.ncbi.nlm.nih.gov',
        'dgv' => 'http://dgv.tcag.ca/dgv/docs/GRCh37_hg19_variants_2020-02-25.txt',
    };

    return ($cnv_filesHR,$hostsHR);
}

sub setOutDir
{
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime();
    my $nowDate = eval($year+1900) . "_" . eval($mon+1) . "_" . $mday;
    my $outDir = defined $odir ? "$odir/CNVdb\_$nowDate/" : "./CNVdb\_$nowDate";
    mkdir $outDir if !-d $outDir;

    my $decipherDir = "$outDir/decipher";
    my $dbvarDir    = "$outDir/dbvar";
    my $dgvDir      = "$outDir/DGV";
    my $clingeneDir = "$outDir/clingene";

    mkdir $decipherDir if !-d $decipherDir;
    mkdir $dbvarDir    if !-d $dbvarDir;
    mkdir $dgvDir      if !-d $dgvDir;
    mkdir $clingeneDir if !-d $clingeneDir;

    return ($decipherDir,$dbvarDir,$dgvDir,$clingeneDir);
}

sub download_clingene
{
    my ($hostsHR, $cnv_filesHR, $clingeneDir) = @_;

    my $geneHTScoreFile = $cnv_filesHR->{'geneHTScore'};
    my $iscaFile = $cnv_filesHR->{'isca'};

    my $rawdataDir = "$clingeneDir/rawDB";
    mkdir $rawdataDir if !-d $rawdataDir;


    my $raw_name_HT   = (split /\//,$geneHTScoreFile)[-1];
    my $raw_name_isca = (split /\//,$iscaFile)[-1];

    system("wget -c -t 0 --timeout 120 $geneHTScoreFile -O $rawdataDir/$raw_name_HT");
    system("wget -c -t 0 --timeout 120 $iscaFile -O $rawdataDir/$raw_name_isca");
    
#    if ($raw_name =~ /gz$/){
#
#        my $flag_gz = system("gunzip $nstdDir/$raw_name");
#        warn "can not uncompree $raw_name!!!\n" if ($flag_gz);
#    }

    #解析clingene文件
    parse_clingene($clingeneDir, $rawdataDir, $raw_name_HT, $raw_name_isca);
    return 0;
}

sub parse_clingene
{
    my ($clingeneDir, $rawdataDir, $geneHTScoreFile, $iscaFile) = @_;

    open HT,"$rawdataDir/$geneHTScoreFile" or die "can not open $rawdataDir/$geneHTScoreFile\n";
    open OHT,">$clingeneDir/$geneHTScoreFile" or die "can not open $clingeneDir/$geneHTScoreFile\n";
    while(<HT>){

        chomp;
        if (/^#/){
            if (!/^#Gene Symbol/){
                next;
            }else{
                print OHT "$_\n";
            }
        }else{
            print OHT "$_\n";
        }
    }

    close HT;
    close OHT;

    open ISCA,"$rawdataDir/$iscaFile" or die "can not open $rawdataDir/$iscaFile\n";
    open OISC,">$clingeneDir/$iscaFile" or die "can not open $clingeneDir/$iscaFile\n";
    while(<ISCA>){

        chomp;
        if (/^#/){
            if (!/^#ISCA ID/){
                next;
            }else{
                print OISC "$_\n";
            }
        }else{
            print OISC "$_\n";
        }      

    }

    close ISCA;close OISC;

    return 0;
}

sub download_dgv
{
    my ($hostsHR, $cnv_filesHR,$dgvDir) = @_;

    my $dgvurl = $hostsHR->{'dgv'};
    my $rawDgvName = (split /\//,$dgvurl)[-1];
    my $rawDgvDir = "$dgvDir/rawDB";
    mkdir $rawDgvDir if !-d $rawDgvDir;

    my $dgvName = $cnv_filesHR->{'dgv'};

    eval{
        system("wget -c -t 0 --timeout 120 $dgvurl -O $rawDgvDir/$rawDgvName");
    };
    warn "can not download $dgvName\n" if $@;

    system("cp $rawDgvDir/$rawDgvName $dgvDir/$dgvName");
    return 0;
}


sub download_dbvar
{
    my ($hostsHR,$cnv_filesHR,$dbvarDir) = @_;

    #my $ftp = Net::FTP->new($hostsHR->{'dbvar'}, Timeout=>240, Debug => 0) or die "Cannot connect to : $@";
    #$ftp->login("anonymous",'-anonymous@')  or die "Cannot login ", $ftp->message;

    #$ftp->cwd("/pub/dbVar/data/Homo_sapiens/by_study/csv/") or die "Cannot change working directory ", $ftp->message;
	
    #循环下载
    my $nstdDir = "$dbvarDir/nstd";
    mkdir $nstdDir if !-d $nstdDir;

    foreach my $fileUrl (values %{$cnv_filesHR->{'dbvarURL'}}){

        my $raw_name = (split /\//,$fileUrl)[-1];
        
        system("wget -c -t 0 --timeout 120 $fileUrl -O $nstdDir/$raw_name");

        if ($raw_name =~ /gz$/){

            my $flag_gz = system("gunzip $nstdDir/$raw_name");
            warn "can not uncompree $raw_name!!!\n" if ($flag_gz);
        }

    }
    #do {$ftp->get($_,"$nstdDir/$_") or die "get failed "} foreach values %{$cnv_filesHR->{'dbvar'}};

    #下载mapping
    #$ftp->cwd("/pub/dbVar/data/release_notes/");
    #my $mappingFile = $cnv_filesHR->{'dbvar_mapping'};
    #$ftp->get($mappingFile,"$nstdDir/$mappingFile") or die "get failed";

    #$ftp->quit;

    #解析nstd文件
    parse_dbvar($dbvarDir, $nstdDir, $cnv_filesHR);
    return 0;
}

sub parse_dbvar
{

    my($dbvarDir,$nstdDir,$cnv_filesHR) = @_;

    my $oldCallHR = {};
    my $oldRegionHR = {};

    ##old study      old call        nstd102 call    old region      nstd102 region
    my ($mapFile) = $cnv_filesHR->{'dbvar_mapping'};

    open MAP,"$nstdDir/$mapFile" or die "$!\n";
    while (<MAP>){
        chomp;
        next if /^#/;
        #old study      old call        nstd102 call    old region      nstd102 region
        #next if /^estd/;
        my ($old_study, $old_call, $nstd102_call, $old_region, $nstd102_region) = split /\s+/,$_;
        $oldCallHR->{"$old_study\t$old_call"} =1 ;
        $oldRegionHR->{"$old_study\t$old_region"} = 1;
    }
    close MAP;

    open OUT,">$dbvarDir/dbVar.GRCh37.all.txt" or die "can not open dbVar.GRCh37.all.txt\n";
    my $nstdFile102 = $cnv_filesHR->{'dbvar'}->{'nstd102'};
    open TD102,"$nstdDir/$nstdFile102" or die;
    while (<TD102>){

        chomp;
        if(/^Study ID/){

            my $header = $_;
            $header =~ s/,/\t/g; print OUT "$header\n";
            next;
        }

        my @oneLine  = split /",/,$_;
        my @newLine = map{ my $t = $_; $t=~ s/"//;$t}@oneLine;
        next if ($newLine[11] !~ /GRCh37/);
        my $lineStr = join "\t",@newLine;
        print OUT "$lineStr\n";
    }
    close TD102;

    foreach my $file($cnv_filesHR->{'dbvar'}->{'nstd37'},$cnv_filesHR->{'dbvar'}->{'nstd45'},$cnv_filesHR->{'dbvar'}->{'nstd51'},$cnv_filesHR->{'dbvar'}->{'nstd101'}){

        open NSTD,"$nstdDir/$file" or die "can not open $file\n";
        while(<NSTD>){

            chomp;
            next if /^Study ID/;

            my @oneLine  = split /",/, $_;
            my @newLine = map{ my $t = $_; $t=~ s/"//;$t}@oneLine;
            next if ($newLine[11] !~ /GRCh37/);
            next if exists ($oldCallHR->{"$newLine[0]\t$newLine[1]"});
            next if exists ($oldRegionHR->{"$newLine[0]\t$newLine[1]"});

            my $lineStr = join "\t", @newLine;
            print OUT "$lineStr\n";

        }
        close NSTD;
    }

    return 0;
}

sub USAGE {#
        my $usage=<<"  .USAGE";

        Description:
                        v1.0:        下载并生成cnv注释需要的文件
        Usage:

                        -all         [options: downing all database]

                        -dgv         [options：download dgv database]
                        -decipher    [options：download decipher database]
                        -dbvar       [options：download dbvar database]
                        -clingene    [options：download clingene database]
                        -odir        [options: default value is './' ]

        Other parameter:
                        -h           Help document


========================================================================== 
Ex: ./cnvDBUpdte_v1.pl  -all 2>log.err
or
Ex: ./Gene_id_match.pl  -dgv -decipher -dbvar -clingene -odir ./ 2>log.err

  .USAGE
        print $usage;
        exit;
}

__END__

